import math
import re
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import time
import pymysql
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests
import pandas as pd

# 目标网址：https://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time=2023%3A01%3A01&end_time=2023%3A12%3A31&timeType=6&displayZone=%E5%8C%97%E4%BA%AC&zoneId=11&pppStatus=0&agentName=
# 设置数据库连接配置
db_config = {
    'host': 'localhost',
    'user': 'gxin',
    'password': 'gxin123',
    'database': 'gxin',
    'charset': 'utf8mb4'
}

def insert_detailed_info(data_list, region_name):
    connection = pymysql.connect(**db_config)
    try:
        with connection.cursor() as cursor:
            inserted_count = 0
            for data in data_list:
                # 打印 data 列表，检查其长度
                print(f"原始数据: {data} (长度: {len(data)})")

                # 先删除多余的重复的hyperlink数据
                delete_sql = """
                DELETE FROM zfbider_23_info
                WHERE hyperlink = %s
                """
                cursor.execute(delete_sql, (data[2],))  # data[2] 是 hyperlink

                # 然后再插入数据
                insert_sql = """
                INSERT INTO zfbider_23_info (time, title, hyperlink, project_number, project_name, award_unit, award_amount, region)
                VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
                ON DUPLICATE KEY UPDATE
                title = VALUES(title),
                hyperlink = VALUES(hyperlink),
                project_number = VALUES(project_number),
                project_name = VALUES(project_name),
                award_unit = VALUES(award_unit),
                award_amount = VALUES(award_amount),
                region = VALUES(region)
                """
                # 直接使用 data 列表
                cursor.execute(insert_sql, data)
                inserted_count += 1  # 统计插入成功的记录数量

        connection.commit()
        if data_list:
            print(f"{region_name}地区，日期 {data_list[0][0]} 的数据已成功存储到数据库，共存储 {inserted_count} 条记录。")
    finally:
        connection.close()

# User-Agent 池
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 11; SM-G996B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Mobile Safari/537.36 Edg/127.0.0.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:116.0) Gecko/20100101 Firefox/116.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 OPR/84.0.0.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 11; SM-G996B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
    "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
    "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 10; SM-A505FN) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
    "Mozilla/5.0 (Linux; U; Android 8.1.0; en-us; Nexus 6P Build/OPM6.171019.030.H1) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/107.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
    # 新增User-Agent
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 9; SM-G950F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36 Edg/130.0.0.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:90.0) Gecko/20100101 Firefox/90.0",
    "Mozilla/5.0 (Linux; Android 8.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/90.0.0.0",
    "Mozilla/5.0 (Linux; Android 7.1.1; Nexus 5X Build/NMF26Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
    "Mozilla/5.0 (Linux; U; Android 10; en-us; Pixel 2 Build/QQ1A.200105.002) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 14_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 11; SM-A515F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; Trident/7.0; AS; rv:12.0) like Gecko",
    "Mozilla/5.0 (Linux; Android 7.1.1; SM-J530F Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 10; SM-N950F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 13_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 11; SM-A750F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 12; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:118.0) Gecko/20100101 Firefox/118.0",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
    "Mozilla/5.0 (iPad; CPU OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 12; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0",
    "Mozilla/5.0 (Linux; Android 10; SM-N975F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 11; SM-A217F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 12; SM-G780F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 13; SM-G990B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.7 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 11; SM-A326B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 16_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.8 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 12; SM-A725F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/158.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/159.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 11; SM-M325FV) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/160.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/161.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/162.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPad; CPU OS 16_9 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.9 Mobile/15E148 Safari/605.1.15",
    "Mozilla/5.0 (Linux; Android 13; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/163.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/164.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/19.0 Safari/605.1.15",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 12; SM-A405FN) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/165.0.0.0 Mobile Safari/537.36"
]
# 设置全局变量来跟踪当前使用的 User-Agent
current_user_agent_index = 0

# 更新全局 current_user_agent_index 并返回新的 User-Agent
def update_user_agent():
    global current_user_agent_index
    current_user_agent_index = (current_user_agent_index + 1) % len(user_agents)
    return user_agents[current_user_agent_index]

# 定义代理 IP 和端口
# proxy_ip = "111.1.61.47"
# proxy_port = "3128"
# proxy = f"{proxy_ip}:{proxy_port}"



# 初始化Selenium WebDriver
def init_selenium():
    chrome_options = Options()
    chrome_options.binary_location = "D:\\google\\chrome-win64\\chrome-win64\\chrome.exe"  # 根据系统实际情况调整路径
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    # 设置ChromeDriver的路径
    service = Service('D:\\anaconda\\chromedriver')  # 这里指定了你安装的ChromeDriver的路径
    # 设置代理
    # chrome_options.add_argument(f'--proxy-server=http://{proxy}')
    # 使用全局的 current_user_agent_index 来设置 User-Agent
    chrome_options.add_argument(f"user-agent={user_agents[current_user_agent_index]}")

    driver = webdriver.Chrome(service=service, options=chrome_options)

    return driver# 使用Selenium获取内容的函数

# 使用Selenium获取内容的函数
def fetch_with_selenium(driver, url, region_name):
    driver.get(url)
    time.sleep(2)  # 等待页面加载
    # 从BeautifulSoup对象中提取其他项目信息
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    project_id, project_name = extract_project_details(soup)
    # 使用第一种方法来提取供应商名称和中标金额
    supplier_name = extract_supplier_names(driver)
    bid_amount = extract_bid_amounts(driver)
    # 如果未能获取供应商名称和中标金额，尝试使用requests方式获取
    if not supplier_name or not bid_amount:
        supplier_name, bid_amount = fetch_with_requests(url, region_name)  # 这里增加 region_name 参数
    return project_id, project_name, supplier_name, bid_amount

# 首先使用requests从网页中提取供应商名称和中标金额
def fetch_with_requests(url, region_name, max_attempts=10):
    global current_user_agent_index
    for attempt in range(max_attempts):
        headers = {
            'User-agent': user_agents[current_user_agent_index],
            'Referer': 'https://www.ccgp.gov.cn/',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
        }
        try:
            response = requests.get(url, headers=headers)
            response.encoding = 'utf-8'
            soup = BeautifulSoup(response.text, 'html.parser')
            supplier_name = None
            bid_amount = None
            for p in soup.find_all('p'):
                text = p.get_text(strip=True)
                if '供应商名称' in text:
                    supplier_name = text.replace('供应商名称：', '').strip()
                elif '中标（成交）金额' in text:
                    bid_amount = text.replace('中标（成交）金额：', '').replace('人民币', '').replace('万元', '').strip()
            if not bid_amount:
                for p in soup.find_all('p'):
                    text = p.get_text(strip=True)
                    if '中标成交供应商名称：' in text:
                        supplier_name = text.replace('中标成交供应商名称：', '').strip()
                    elif '总中标成交金额' in text:
                        bid_amount = text.replace('总中标成交金额', '').replace('（人民币）', '').replace('万元', '').strip()

            if bid_amount:
                bid_amount = clean_bid_amount(bid_amount)
            # 如果成功获取到数据，返回结果
            if supplier_name or bid_amount:
                return supplier_name, bid_amount
        except requests.exceptions.ConnectionError as e:
            print(f"连接失败: {e}，等待5秒后重试... (尝试 {attempt + 1}/{max_attempts})")
            time.sleep(5)
        # 如果当前 User-Agent 无效，则尝试下一个
        update_user_agent()
    # 如果所有尝试都失败，返回 None
    return None, None
#当使用request库失败时则修改为用selenium库实现
# 从BeautifulSoup对象中提取项目编号和项目名称
def extract_project_details(soup):
    project_id = ''
    project_name = ''
    # 优先使用方法1中的方法从BeautifulSoup对象中提取信息
    project_id_element = soup.find(text=re.compile(r'一、项目编号[:：]'))
    if project_id_element:
        project_id = re.sub(r'一、项目编号[:：]\s*', '', project_id_element.parent.text.strip())
    project_name_element = soup.find(text=re.compile(r'二、项目名称[:：]'))
    if project_name_element:
        project_name = re.sub(r'二、项目名称[:：]\s*', '', project_name_element.parent.text.strip())
    # 如果未能提取到信息，使用方法2的方法
    if not project_id or not project_name:
        paragraphs = soup.find_all('p')
        for p in paragraphs:
            text = p.get_text(strip=True)
            if '项目编号' in text and not project_id:
                project_id = text.replace('1．项目编号：', '').strip()
            if '项目名称' in text and not project_name:
                project_name = text.replace('2．项目名称：', '').strip()
    # 进一步使用方法3的正则表达式匹配方法
    page_source = str(soup)
    if not project_id:
        project_id_element = re.search(r'一、项目编号[:：]\s*([^\s<]*)', page_source)
        if project_id_element:
            project_id = project_id_element.group(1).strip()

    if not project_name:
        project_name_element = re.search(r'二、项目名称[:：]\s*([^\s<]*)', page_source)
        if project_name_element:
            project_name = project_name_element.group(1).strip()
    return project_id, project_name
# 清理项目编号和项目名称，保留冒号后的部分
def clean_project_details(project_id, project_name):
    project_id_cleaned = re.sub(r'.*[:：]\s*', '', project_id)
    project_name_cleaned = re.sub(r'.*[:：]\s*', '', project_name)
    return project_id_cleaned, project_name_cleaned

#使用selenium获取供应商信息
def extract_supplier_names(driver):
    try:
        supplier_name = None
        # 首先尝试从表格中提取供应商名称
        supplier_name = extract_from_table(driver, '供应商名称')
        if supplier_name:
            return supplier_name
        # 如果表格中未找到供应商名称，继续使用代码1的方法
        tables = driver.find_elements(By.TAG_NAME, "table")
        for table in tables:
            rows = table.find_elements(By.TAG_NAME, "tr")
            # 初始化变量存储供应商名称和中标金额列的索引
            supplier_name_index = None
            for i, row in enumerate(rows):
                cells = row.find_elements(By.TAG_NAME, "td")
                if supplier_name_index is None:
                    # 遍历每个单元格寻找“供应商名称”字段
                    for j, cell in enumerate(cells):
                        if "供应商名称" in cell.text:
                            supplier_name_index = j
                # 如果已经找到供应商名称的索引
                if supplier_name_index is not None and i > 0:
                    supplier_td = cells[supplier_name_index]
                    # 确保输出的不是表头
                    if "供应商名称" not in supplier_td.text:
                        return supplier_td.text.strip()
        # 如果表格内仍未找到，尝试通过直接查找文本
        supplier_name_tag = driver.find_element(By.XPATH, "//*[contains(text(),'供应商名称：')]")
        if supplier_name_tag:
            supplier_name = supplier_name_tag.text.strip().split("：")[-1].strip()
            return supplier_name
        # 如果仍然未找到供应商名称，返回 "未找到供应商名称"
        return "未找到供应商名称"
    except Exception as e:
        return None
#使用selenium获取中标金额数据
def extract_bid_amounts(driver):
    try:
        # 尝试从表格中提取中标金额
        bid_amount = extract_from_table(driver, '中标金额')
        if bid_amount:
            return clean_bid_amount(bid_amount)
        # 尝试通过 XPath 提取 "中标金额：" 格式的金额
        try:
            bid_amount_tag_1 = driver.find_element(By.XPATH, "//*[contains(text(),'中标金额：')]")
            if bid_amount_tag_1:
                bid_amount_1 = bid_amount_tag_1.text.strip().split("：")[-1].strip()
                if bid_amount_1:
                    return clean_bid_amount(bid_amount_1)
        except Exception:
            pass
        # 尝试通过 XPath 提取 "中标（成交）金额：" 格式的金额
        try:
            bid_amount_tag_2 = driver.find_element(By.XPATH, "//*[contains(text(),'中标（成交）金额：')]")
            if bid_amount_tag_2:
                bid_amount_2 = bid_amount_tag_2.text.strip().split("：")[-1].strip()
                if bid_amount_2:
                    return clean_bid_amount(bid_amount_2)
        except Exception:
            pass
        # 如果以上方法都未找到，使用新的方法再次尝试
        tables = driver.find_elements(By.TAG_NAME, "table")
        for table in tables:
            rows = table.find_elements(By.TAG_NAME, "tr")
            supplier_name_index = None
            bid_amount_index = None
            for i, row in enumerate(rows):
                cells = row.find_elements(By.TAG_NAME, "td")
                if supplier_name_index is None or bid_amount_index is None:
                    for j, cell in enumerate(cells):
                        if "中标金额" in cell.text:
                            bid_amount_index = j
                            break
                if bid_amount_index is not None and i > 0:
                    bid_amount_td = cells[bid_amount_index]
                    if "中标金额" not in bid_amount_td.text:
                        bid_amount_str = bid_amount_td.text.strip().replace(",", "")
                        try:
                            bid_amount = float(bid_amount_str)
                            bid_amount /= 10000
                            return f"{bid_amount:,.4f}"
                        except ValueError:
                            return "格式错误"
        return "未找到中标金额"
    except Exception:
        return None
#清洗中标金额数据
def clean_bid_amount(bid_amount):
    # 使用正则表达式去除所有括号及括号内的内容（包括中文括号和英文括号）
    cleaned_amount = re.sub(r'[\(\（][^\)\）]*[\)\）]', '', bid_amount)  # 删除括号及其内容
    # 删除所有非数字和小数点的字符
    cleaned_amount = re.sub(r'[^\d.]', '', cleaned_amount)

    try:
        # 转换为浮点数并保留小数点后四位
        amount_float = float(cleaned_amount)
        formatted_amount = f"{amount_float:.4f}"  # 保留小数点后四位，补齐零
        return formatted_amount
    except ValueError:
        # 如果转换失败，返回清理后的金额
        return cleaned_amount

# 从表格中提取信息的通用函数
def extract_from_table(driver, column_name):
    try:
        WebDriverWait(driver, 2).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#projectBundleList'))
        )

        table = driver.find_element(By.CSS_SELECTOR, '#projectBundleList')

        header = table.find_element(By.CSS_SELECTOR, 'tbody > tr.tableHeader')
        header_columns = header.find_elements(By.TAG_NAME, 'td')

        column_index = -1

        for index, column in enumerate(header_columns):
            if re.search(column_name, column.text.strip()):
                column_index = index
                break

        if column_index == -1:
            return None

        rows = table.find_elements(By.CSS_SELECTOR, 'tbody > tr')

        for row in rows[1:]:
            columns = row.find_elements(By.TAG_NAME, 'td')
            return columns[column_index].text.strip()

        return None

    except Exception:
        return None

# 初始化计数器
record_count = 0

# 抓取详细信息，使用Selenium和Requests
def fetch_project_details(url, region_name):
    global current_user_agent_index
    max_attempts = len(user_agents)  # 尝试次数等于 User-Agent 数量
    for attempt in range(max_attempts):
        driver = init_selenium()
        project_id, project_name, supplier_name, bid_amount = fetch_with_selenium(driver, url, region_name)
        driver.quit()

        if project_id and project_name and supplier_name and bid_amount:
            break
        else:
            # 如果未能成功抓取，更新 User-Agent 并重试
            update_user_agent()

    # 清理项目编号和项目名称，保留冒号后的部分
    project_id, project_name = clean_project_details(project_id, project_name)

    return project_id, project_name, supplier_name, bid_amount, region_name
# 打印信息到控制台
def print_info(title, link, date_text, project_id, project_name, supplier_name, bid_amount ,region_name):
    print(f"标题: {title}")
    print(f"链接: {link}")
    print(f"时间: {date_text}")
    print(f"项目编号: {project_id}")
    print(f"项目名称: {project_name}")
    print(f"供应商名称: {supplier_name}")
    print(f"中标（成交）金额: {bid_amount}")
    print(f"地域: {region_name}")
    print("-" * 50)

# 设置抓取的地区
regions = [
    ("天津", "12")
    # ("北京", "11"),
    # ("河北", "13")
]

start_date = "2024-10-14"
end_date = "2024-10-15"
# 河北和北京的 2022-12-05开始停止爬取
# 时间循环，倒序爬取
date_range = pd.date_range(end=end_date, start=start_date).strftime('%Y:%m:%d')[::-1]

# 主循环，抓取不同日期和地区的项目数据
for current_date in date_range:
    for region_name, zone_id in regions:
        page_index = 1
        page_size = 20
        stop_crawling = False
        data_list = []
        record_count = 0
        total_pages = 1  # 初始化总页数

        while not stop_crawling:
            url = f'https://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index={page_index}&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time={current_date}&end_time={current_date}&timeType=6&displayZone={region_name}&zoneId={zone_id}&pppStatus=0&agentName='

            driver = init_selenium()
            driver.get(url)
            time.sleep(1)
            page_content = driver.page_source
            soup_obj = BeautifulSoup(page_content, 'html.parser')

            # 抓取总记录数
            try:
                total_records_element = soup_obj.select_one(
                    'div > p> span:nth-child(2)')
                if total_records_element:
                    total_records = int(total_records_element.text.strip())  # 确保将文本转换为整数
                    total_pages = math.ceil(total_records / page_size)
                    print(f"{region_name} 地区 {current_date} 日期的总数据量: {total_records}, 总页数: {total_pages}")
                else:
                    print(f"未找到 {region_name} 地区 {current_date} 日期的总数据量信息")
            except Exception as e:
                print(f"获取总记录数时发生错误: {e}")
            elements = soup_obj.select(
                'body > div:nth-child(11) > div.vT_z > div > div > div.vT-srch-result-list > ul > li > a')
            time_elements = soup_obj.select(
                'body > div:nth-child(11) > div.vT_z > div > div > div.vT-srch-result-list > ul > li > span')

            # 处理当前页的数据
            for element, time_element in zip(elements, time_elements):
                if element is None:
                    continue

                title = element.text.strip()
                link = element['href']
                time_text = time_element.text.strip()
                date_text = time_text.split()[0].replace('.', '-')

                project_id, project_name, supplier_name, bid_amount, region = fetch_project_details(link, region_name)

                if supplier_name is None:
                    supplier_name = "未找到供应商名称"
                if bid_amount is None:
                    bid_amount = "未找到中标金额"

                print_info(title, link, date_text, project_id, project_name, supplier_name, bid_amount, region_name) #打印爬取情况
                data_list.append((date_text, title, link, project_id, project_name, supplier_name, bid_amount, region))
                record_count += 1

                # 每30条数据存储一次
                if record_count >= 1:
                    insert_detailed_info(data_list, region_name)
                    data_list = []
                    record_count = 0  # 重置计数器

            # 判断是否到达最后一页
            if page_index >= total_pages:
                stop_crawling = True
            driver.quit()
            page_index += 1  # 确保每次循环后page_index递增

        # 日期处理结束后，插入剩余数据
        if data_list:
            insert_detailed_info(data_list, region_name)
            data_list = []
#更新于0902 以验证不足20条会进行 满足也会 并且增加去重操作
